{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 数据预处理"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "#2021/11/25\n",
    "#张赛赛 \n",
    "#pv_deal"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 过滤\n",
    "非蒙阴用户，\n",
    "\n",
    "时间序列不全用户（5-20），\n",
    "\n",
    "发电为0用户，\n",
    "\n",
    "容量为0用户，\n",
    "\n",
    "编号重复用户（多个用户同一个编号的情况）\n",
    "### 填充\n",
    "发电为空，\n",
    "\n",
    "容量为空，\n",
    "\n",
    "### 添加\n",
    "发电功率（每小时内发电量新增量/光伏容量）\n",
    "\n",
    "### 排序\n",
    "\n",
    "按用户分组排序"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 数据过滤"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "691d4fdb19de4e839707798a89584009",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=17.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "20211109 可用数据条数: 2048 ,数量%16 = 0\n",
      "20211110 可用数据条数: 1872 ,数量%16 = 0\n",
      "20211113 可用数据条数: 2256 ,数量%16 = 0\n",
      "20211114 可用数据条数: 144 ,数量%16 = 0\n",
      "20211115 可用数据条数: 2256 ,数量%16 = 0\n",
      "20211117 可用数据条数: 2192 ,数量%16 = 0\n",
      "20211118 可用数据条数: 2240 ,数量%16 = 0\n",
      "20211119 可用数据条数: 2208 ,数量%16 = 0\n",
      "20211120 可用数据条数: 2224 ,数量%16 = 0\n",
      "20211121 可用数据条数: 2192 ,数量%16 = 0\n",
      "20211122 可用数据条数: 2144 ,数量%16 = 0\n",
      "20211123 可用数据条数: 2224 ,数量%16 = 0\n",
      "20211124 可用数据条数: 2208 ,数量%16 = 0\n",
      "20211125 可用数据条数: 2192 ,数量%16 = 0\n",
      "20211126 可用数据条数: 2208 ,数量%16 = 0\n",
      "20211129 可用数据条数: 1648 ,数量%16 = 0\n",
      "20211130 可用数据条数: 2160 ,数量%16 = 0\n",
      "\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import json\n",
    "import copy\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "def read_label_data(path):\n",
    "    #读入最小数据\n",
    "    #print('标签数据：',path[-35:])\n",
    "    with open(path,encoding='utf-8') as f:\n",
    "        content = f.read().replace(\"\\n\",\"\")\n",
    "    content = content[0:-1]\n",
    "    temp = '{\"res\":[' + content + ']}'\n",
    "    json_data = json.loads(temp)\n",
    "    #print(\"总用户数:\",len(json_data['res']))\n",
    "    return json_data\n",
    "\n",
    "def filter_data(path,json_data):\n",
    "    #过滤非蒙阴县台区\n",
    "    sbbm = np.loadtxt('./data/37413_1.txt', dtype = str)\n",
    "    filterdata = []\n",
    "    for i in range(len(json_data['res'])):\n",
    "        data = dict()\n",
    "        data['sbbm'] = json_data['res'][i]['SBBM']\n",
    "        data['ycsb'] = json_data['res'][i]['result']['Data']['ycsb']\n",
    "        data['data_date'] = json_data['res'][i]['result']['Data']['data_date']\n",
    "        data['yhmc'] = json_data['res'][i]['result']['Data']['yhmc']\n",
    "        data['gfrl'] = float(json_data['res'][i]['result']['Data']['gfrl']) if json_data['res'][i]['result']['Data']['gfrl']!=None else 0\n",
    "        data['drfdl'] = float(json_data['res'][i]['result']['Data']['drfdl']) if json_data['res'][i]['result']['Data']['drfdl']!=None else 0\n",
    "        #data['dwfd'] = data['drfdl']/data['gfrl'] if data['gfrl']!=0 else 0\n",
    "        #过滤蒙阴sbbm \n",
    "        #过滤drfdl为0/null\n",
    "        #过滤gfrl为0/null\n",
    "        if path[-8:]=='20211110':\n",
    "            if((data['sbbm'] in sbbm) and \n",
    "               data['drfdl'] != 0 and     \n",
    "               data['gfrl']!= 0 and \n",
    "               data['ycsb']!='202106190019GF'and  #编号重复\n",
    "               data['ycsb']!='202106160265GF'and  #数值偏大\n",
    "               data['ycsb']!='202106160268GF'and\n",
    "               data['ycsb']!='202106160298GF'and\n",
    "               data['ycsb']!='202106160299GF'):\n",
    "                filterdata.append(data)\n",
    "        else:\n",
    "            if((data['sbbm'] in sbbm) and \n",
    "               data['drfdl'] != 0 and     \n",
    "               data['gfrl']!= 0 and \n",
    "               data['ycsb']!='202106190019GF'):\n",
    "                filterdata.append(data)\n",
    "    #转为dataframe\n",
    "    filterdata = pd.DataFrame(filterdata)\n",
    "    #print(\"过滤后可用蒙阴县用户数:\",len(filterdata))\n",
    "    return filterdata['ycsb'].tolist()\n",
    "    \n",
    "def get_data(path,filterdata):\n",
    "    tempfilter = copy.deepcopy(filterdata)#深拷贝过滤列表\n",
    "    with open(path,encoding='utf-8') as f:\n",
    "        content = f.read().replace(\"\\n\",\"\")\n",
    "    content = content[0:-1]\n",
    "    temp = '{\"res\":[' + content + ']}'\n",
    "    json_data = json.loads(temp)\n",
    "    res = []\n",
    "    for i in range(len(json_data['res'])):\n",
    "        data = dict()\n",
    "        data['sbbm'] = json_data['res'][i]['SBBM']\n",
    "        data['ycsb'] = json_data['res'][i]['result']['Data']['ycsb']\n",
    "        data['data_date'] = path[-20:-12] + path[-11:-9]\n",
    "        data['time'] = path[-20:-12]\n",
    "        data['yhmc'] = json_data['res'][i]['result']['Data']['yhmc']\n",
    "          \n",
    "    #异常标记\n",
    "        #光伏容量\n",
    "        data['gfrl'] = float(json_data['res'][i]['result']['Data']['gfrl'])  if json_data['res'][i]['result']['Data']['gfrl']!=None else 999 \n",
    "        #累计发电量\n",
    "        data['drfdl'] = float(json_data['res'][i]['result']['Data']['drfdl'])  if json_data['res'][i]['result']['Data']['drfdl']!=None else 999\n",
    "        #vol_a为空\n",
    "        data['vol_a'] = json_data['res'][i]['result']['Data']['vol_a'] if json_data['res'][i]['result']['Data']['vol_a']!=None else 999\n",
    "        #vol_b为空\n",
    "        data['vol_b'] = json_data['res'][i]['result']['Data']['vol_b'] if json_data['res'][i]['result']['Data']['vol_b']!=None else 999\n",
    "        #vol_c为空\n",
    "        data['vol_c'] = json_data['res'][i]['result']['Data']['vol_c'] if json_data['res'][i]['result']['Data']['vol_c']!=None else 999\n",
    "        #电流具有周期性和发电功率等类似\n",
    "        data['cur_a'] = json_data['res'][i]['result']['Data']['cur_a'] if json_data['res'][i]['result']['Data']['cur_a']!=None else 999\n",
    "        data['cur_b'] = json_data['res'][i]['result']['Data']['cur_b'] if json_data['res'][i]['result']['Data']['cur_b']!=None else 999\n",
    "        data['cur_c'] = json_data['res'][i]['result']['Data']['cur_c'] if json_data['res'][i]['result']['Data']['cur_c']!=None else 999\n",
    "        data['p'] = json_data['res'][i]['result']['Data']['p'] if json_data['res'][i]['result']['Data']['p']!=None else 999\n",
    "        data['q'] = json_data['res'][i]['result']['Data']['q'] if json_data['res'][i]['result']['Data']['q']!=None else 999\n",
    "        data['szgl'] = json_data['res'][i]['result']['Data']['szgl'] if json_data['res'][i]['result']['Data']['szgl']!=None else 999\n",
    "        data['hgl'] = json_data['res'][i]['result']['Data']['hgl'] if json_data['res'][i]['result']['Data']['hgl']!=None else 999\n",
    "        data['glys'] = json_data['res'][i]['result']['Data']['glys'] if json_data['res'][i]['result']['Data']['glys']!=None else 999\n",
    "        #用标签数据过滤\n",
    "        #过滤部分发电异常用户\n",
    "        #用户编号重复用户 \n",
    "        if(data['ycsb'] in tempfilter):\n",
    "            tempfilter.remove(data['ycsb'])\n",
    "            res.append(data)  \n",
    "    #print(path[-20:],\"条数：\",len(res),tempfilter)\n",
    "    return [res,tempfilter]\n",
    "\n",
    "def get_save(path,filename):\n",
    "    path = path + filename\n",
    "    res = []\n",
    "    files= os.listdir(path)\n",
    "    allfilter = []\n",
    "    for file in files:az\n",
    "        if os.path.isdir(file):\n",
    "            for file in os.listdir(path+'/data/'):\n",
    "                json_data = read_label_data(path+'/data/'+file)\n",
    "                filterdata = filter_data(path,json_data)\n",
    "                #print(filterdata)\n",
    "        else:\n",
    "            result = get_data(path + '/' + file,filterdata)\n",
    "            res.extend(result[0])\n",
    "            data = result[1]\n",
    "            for d in data:\n",
    "                allfilter.append(d)\n",
    "    #过滤数据不全用户\n",
    "    test = []\n",
    "    for r in res:\n",
    "         if(r['ycsb'] in allfilter):\n",
    "                res.remove(r)\n",
    "                test.append(r)\n",
    "    pvdata = pd.DataFrame(res)\n",
    "    #print(\"allfilter\",allfilter)\n",
    "    #print(\"test\",test)\n",
    "    \n",
    "    #按用户分组排序\n",
    "    pvdata = pd.DataFrame(pvdata.sort_values(by=['ycsb','data_date']))\n",
    "    #重置index\n",
    "    pvdata.reset_index(drop=True,inplace=True)\n",
    "    bzw = 1\n",
    "    for i in range(int(len(pvdata)/16)):\n",
    "        if(pvdata[\"drfdl\"][(i+1)*16-1]<=pvdata[\"drfdl\"][i*16]):\n",
    "            bzw = -1\n",
    "        for j in range(16):\n",
    "            if(j>0):\n",
    "                if(pvdata[\"drfdl\"][i*16+j]<pvdata[\"drfdl\"][i*16+j-1]):\n",
    "                    #print('异常数据',pvdata[\"yhmc\"][i*16],pvdata[\"drfdl\"][i*16+j-1],'***',pvdata[\"drfdl\"][i*16+j])\n",
    "                    bzw = -1\n",
    "        if(bzw == -1):\n",
    "#             print(pvdata[\"yhmc\"][i*16])\n",
    "            for k in range(16):\n",
    "#                 print(pvdata[\"drfdl\"][i*16+k])\n",
    "                pvdata.drop(index = i*16+k,inplace=True)\n",
    "                bzw = 1\n",
    "    print(filename,\"可用数据条数:\",len(pvdata),',数量%16 =',len(pvdata)%16)\n",
    "    pvdata.reset_index(drop=True,inplace=True)\n",
    "    #查询数据异常文件，时间序列不够16的用户\n",
    "#     if(filename == '20211122'):\n",
    "#         test = 0\n",
    "#         name = ''\n",
    "#         old_list = []\n",
    "#         old_list = set()\n",
    "#         new_list = []\n",
    "#         for i in range(len(pvdata)):\n",
    "#             old_list.add(pvdata[\"yhmc\"][i]) \n",
    "#             new_list.append(pvdata[\"yhmc\"][i]) \n",
    "#         new_old_list = list(old_list)\n",
    "#         for j in range(len(new_old_list)):\n",
    "#             if(new_list.count(new_old_list[j])!=16):\n",
    "#                 print(new_old_list[j],new_list.count(new_old_list[j]))\n",
    "    #保存数据\n",
    "    path = './data/filter_data/'\n",
    "    if not os.path.exists(path):\n",
    "        os.mkdir(path)\n",
    "    pvdata.to_csv(path + filename + '.csv', encoding='utf_8_sig',index=None)\n",
    "\n",
    "if __name__ == '__main__':\n",
    "    path = './data/pv_data/'\n",
    "    for file in tqdm(os.listdir(path)):\n",
    "        get_save(path,file)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 异常填充"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "合并数据\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8fce155c68754ecd8e169085385f5e3c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value=''), FloatProgress(value=0.0, max=17.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "填补异常数据\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1d362e72958c40e1857389f5fc8dd1af",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value=''), FloatProgress(value=1.0, bar_style='info', layout=Layout(width='20px'), max=1.0…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "数据保存成功\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "from tqdm.notebook import tqdm\n",
    "import os\n",
    "\n",
    "def join_weater(pv_data):\n",
    "    weather_data = './data/weather1.csv'\n",
    "    weather_data = pd.read_csv(weather_data)\n",
    "    pvdata = pd.merge(pv_data, weather_data, how='left', on='time') #光伏数据与天气数据根据时间左连接合并\n",
    "    pvdata.insert(5,'hour',(pvdata['data_date'])%100)\n",
    "    #pvdata = pd.DataFrame(pvdata.sort_values(by=['ycsb','data_date']))\n",
    "    #train_data.to_csv('./data/train_data_test_all/'+file, encoding='utf_8_sig',index=None)\n",
    "    return pvdata\n",
    "\n",
    "def merger_data(pv_data):\n",
    "    print('合并数据')\n",
    "    all_csv_list = os.listdir(pv_data)\n",
    "    for single_csv in tqdm(all_csv_list):\n",
    "        #print(single_csv)\n",
    "        single_data_frame = pd.read_csv(os.path.join(pv_data, single_csv))\n",
    "        if single_csv == all_csv_list[0]:\n",
    "            all_data_frame = join_weater(single_data_frame)\n",
    "        else:\n",
    "            single_data_frame = join_weater(single_data_frame)\n",
    "            all_data_frame = pd.concat([all_data_frame, single_data_frame],axis=0)\n",
    "    all_data_frame= pd.DataFrame(all_data_frame.sort_values(by=['ycsb','data_date'])).reset_index()\n",
    "    all_data_frame.to_csv('./data/merger_data.csv', encoding='utf_8_sig',index=None)\n",
    "    return all_data_frame\n",
    "\n",
    "#获取填充词典\n",
    "def get_fill_dict():\n",
    "    pvdata = pd.read_csv('./data/fill_data.csv', encoding='gbk')\n",
    "    f = pvdata[pvdata['gfrl']==49.455]  #49.455容量是最多的用户装机量，具有一定代表性\n",
    "    # 第一个维度为天气\n",
    "    fill_dict = dict()\n",
    "    # 第二个维度是时间\n",
    "    for weather in ['多云','晴','雨','阴']:\n",
    "        f1 = f[f['weather'] == weather]\n",
    "        d1 = pd.DataFrame(f1.groupby('hour').mean())\n",
    "        d1 = d1.loc[:,'fdgl':'glys']\n",
    "        d1 = d1.T\n",
    "        d1 = d1.to_dict()\n",
    "        fill_dict[weather] = d1\n",
    "    return fill_dict\n",
    "\n",
    "def fill_data(merger_data,fill_dict):\n",
    "    print('填补异常数据')\n",
    "    temp_drfdl = 0\n",
    "    fdgl = []\n",
    "    tempdrfdl = []\n",
    "    for index, data in tqdm(merger_data.iterrows()):\n",
    "        if data['vol_a'] == 0 or data['vol_a'] == 999:\n",
    "            merger_data.loc[index,'vol_a'] = fill_dict[data['weather']][data['hour']]['vol_a']\n",
    "        if data['vol_b'] == 0 or data['vol_b'] == 999:\n",
    "            merger_data.loc[index,'vol_b'] = fill_dict[data['weather']][data['hour']]['vol_b'] \n",
    "        if data['vol_c'] == 0 or data['vol_c'] == 999:\n",
    "            merger_data.loc[index,'vol_c'] = fill_dict[data['weather']][data['hour']]['vol_c']    \n",
    "        if data['cur_a'] == 0 or data['cur_a'] == 999:\n",
    "            merger_data.loc[index,'cur_a'] = fill_dict[data['weather']][data['hour']]['cur_a']    \n",
    "        if data['cur_b'] == 0 or data['cur_b'] == 999:\n",
    "            merger_data.loc[index,'cur_b'] = fill_dict[data['weather']][data['hour']]['cur_b']    \n",
    "        if data['cur_c'] == 0 or data['cur_c'] == 999:\n",
    "            merger_data.loc[index,'cur_c'] = fill_dict[data['weather']][data['hour']]['cur_c']  \n",
    "        if data['p'] ==0 or data['p'] == 999:\n",
    "            merger_data.loc[index,'p'] = fill_dict[data['weather']][data['hour']]['p']  \n",
    "        if data['q'] ==0 or data['q'] == 999:\n",
    "            merger_data.loc[index,'q'] = fill_dict[data['weather']][data['hour']]['q']\n",
    "        if data['szgl'] ==0 or data['szgl'] == 999:\n",
    "            merger_data.loc[index,'szgl'] = fill_dict[data['weather']][data['hour']]['szgl']\n",
    "        if data['hgl'] ==0 or data['hgl'] == 999:\n",
    "            merger_data.loc[index,'hgl'] = fill_dict[data['weather']][data['hour']]['hgl']   \n",
    "        if data['glys'] ==0 or data['glys'] == 999:\n",
    "            merger_data.loc[index,'glys'] = fill_dict[data['weather']][data['hour']]['glys']\n",
    "        #发电功率\n",
    "        if index % 16 == 0:\n",
    "            fdgl.append(0)\n",
    "            temp_drfdl = 0\n",
    "        if index % 16 != 0:\n",
    "            if data['drfdl'] != 999:\n",
    "                if data['drfdl'] >= temp_drfdl:\n",
    "                    fdgl.append((data['drfdl']-temp_drfdl) / data['gfrl'])\n",
    "                    temp_drfdl = data['drfdl']\n",
    "                else:\n",
    "                    fdgl.append(fill_dict[data['weather']][data['hour']]['fdgl'])\n",
    "                    temp_drfdl = fill_dict[data['weather']][data['hour']]['fdgl']* data['gfrl'] + temp_drfdl\n",
    "            else:\n",
    "                #merger_data.loc[index,'drfdl'] = fill_dict[data['weather']][data['hour']]['fdgl'] * data['gfrl'] + temp_drfdl\n",
    "                fdgl.append(fill_dict[data['weather']][data['hour']]['fdgl'])\n",
    "                temp_drfdl = fill_dict[data['weather']][data['hour']]['fdgl'] * data['gfrl'] + temp_drfdl\n",
    "                #fdgl.append((data['drfdl']-temp_drfdl) / data['gfrl'])\n",
    "                #temp_drfdl = data['drfdl']\n",
    "    merger_data.insert(8,'fdgl',fdgl)\n",
    "    \n",
    "    weather_select = './data/weather_data/weather_select_1.csv'\n",
    "    weather_select_data = pd.read_csv(weather_select)\n",
    "    pvdata = pd.merge(merger_data, weather_select_data, how='left', on='data_date')\n",
    "    \n",
    "    pvdata.to_csv('./data/train_data_1.csv', encoding='utf_8_sig',index=None)\n",
    "    print('数据保存成功')\n",
    "    return merger_data\n",
    "\n",
    "if __name__=='__main__':\n",
    "    pv_data = './data/filter_data/'\n",
    "    #拼接天气源1，合并所有日期数据\n",
    "    merger_data = merger_data(pv_data)\n",
    "    #获取填充词典\n",
    "    fill_dict = get_fill_dict()\n",
    "    #按天气分类，填充对应时间的均值\n",
    "    fill_data = fill_data(merger_data,fill_dict)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 天气源1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "添加风速\n",
      "添加温差\n",
      "添加北京时间\n",
      "保存成功 ./data/weather_data/weather_select.csv\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "\n",
    "#读取天气数据\n",
    "def add_fs(weather):\n",
    "    print('添加风速')\n",
    "    fs = abs(weather['北向风速(V,m/s)']) + abs(weather['东向风速(U,m/s)'])\n",
    "    weather.insert(1,'风速',fs)\n",
    "    return weather\n",
    "\n",
    "def add_date(weather):\n",
    "    print('添加北京时间')\n",
    "    time = []\n",
    "    for i in range(weather.shape[0]):\n",
    "        h = weather['时间(UTC)'][i]\n",
    "        y = weather['日期'][i]\n",
    "        if len(h)==4:\n",
    "            hour = str(int(h[0:1])+8) if int(h[0:1])+8>9 else '0'+str(int(h[0:1])+8)\n",
    "            time.append(str(y)+hour)\n",
    "        else:\n",
    "            hour = str(int(h[0:2]) + 8) if int(h[0:2])<=15 else '0'+str(int(h[0:2])-16)\n",
    "            d = str(y) if int(h[0:2])<=15 else str(int(y)+1)\n",
    "            time.append(d+hour)\n",
    "    weather.insert(8,'data_date',time)\n",
    "    return weather\n",
    "\n",
    "def add_t(weather):\n",
    "    print('添加温差')\n",
    "    t = weather['地表温度(℃)']-weather['气温2m(℃)']\n",
    "    weather.insert(5,'风',t)\n",
    "    return weather\n",
    "\n",
    "def select_feature(weather):\n",
    "    weatherfeature = ['地面气压(hPa)','气温2m(℃)','地表温度(℃)','相对湿度(%)','风速','总云量(tcc)','净日照强度(net,J/m2)','data_date']\n",
    "    path = './data/weather_data/weather_select.csv'\n",
    "    weather[weatherfeature].to_csv(path, encoding='utf_8_sig',index=None)\n",
    "    print('保存成功',path)\n",
    "    \n",
    "if __name__ == '__main__':\n",
    "    weather = pd.read_csv('./data/weather_data/35.75.csv')\n",
    "    weather = add_fs(weather)\n",
    "    weather = add_t(weather)\n",
    "    weather = add_date(weather)\n",
    "    weather = select_feature(weather)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "完成合并: ./data/date_data/20211109.csv\n",
      "完成合并: ./data/date_data/20211110.csv\n",
      "完成合并: ./data/date_data/20211113.csv\n",
      "完成合并: ./data/date_data/20211114.csv\n",
      "完成合并: ./data/date_data/20211115.csv\n",
      "完成合并: ./data/date_data/20211117.csv\n",
      "完成合并: ./data/date_data/20211118.csv\n",
      "完成合并: ./data/date_data/20211119.csv\n",
      "完成合并: ./data/date_data/20211120.csv\n",
      "完成合并: ./data/date_data/20211121.csv\n"
     ]
    }
   ],
   "source": [
    "def merge_data(pv_data,weather_data):\n",
    "    weather_data = pd.read_csv(weather_data)\n",
    "    files= os.listdir(pv_data)\n",
    "    for file in files:\n",
    "        path = pv_data + file\n",
    "        pvdata = pd.read_csv(path)\n",
    "        train_data = pd.merge(pvdata, weather_data, how='left', on='data_date')#光伏数据与天气数据根据时间左连接合并\n",
    "        train_data.insert(5,'hour',(train_data['data_date'])%100)\n",
    "        train_data = train_data.fillna(method ='ffill')\n",
    "        train_data.to_csv('./data/train_data/'+file, encoding='utf_8_sig',index=None)\n",
    "        print('完成合并:',path)\n",
    "    return train_data\n",
    "\n",
    "if __name__=='__main__':\n",
    "    pv_data = './data/date_data/'\n",
    "    weather_data = './data/weather_data/weather_select.csv'\n",
    "    train_data = merge_data(pv_data,weather_data)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 天气源2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "ename": "FileNotFoundError",
     "evalue": "[Errno 2] File ./data/weather1.csv does not exist: './data/weather1.csv'",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mFileNotFoundError\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-49-dfe5572316e5>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m     34\u001b[0m     \u001b[0mpv_data\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;34m'./data/date_data_all/'\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     35\u001b[0m     \u001b[0mweather_data\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;34m'./data/weather1.csv'\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 36\u001b[1;33m     \u001b[0mmerge_data\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mpv_data\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mweather_data\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     37\u001b[0m     \u001b[0mread_merger_data\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m<ipython-input-49-dfe5572316e5>\u001b[0m in \u001b[0;36mmerge_data\u001b[1;34m(pv_data, weather_data)\u001b[0m\n\u001b[0;32m      3\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      4\u001b[0m \u001b[1;32mdef\u001b[0m \u001b[0mmerge_data\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mpv_data\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mweather_data\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 5\u001b[1;33m     \u001b[0mweather_data\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mpd\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mread_csv\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mweather_data\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      6\u001b[0m     \u001b[0mfiles\u001b[0m\u001b[1;33m=\u001b[0m \u001b[0mos\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mlistdir\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mpv_data\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      7\u001b[0m     \u001b[1;32mfor\u001b[0m \u001b[0mfile\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mfiles\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\anaconda\\lib\\site-packages\\pandas\\io\\parsers.py\u001b[0m in \u001b[0;36mparser_f\u001b[1;34m(filepath_or_buffer, sep, delimiter, header, names, index_col, usecols, squeeze, prefix, mangle_dupe_cols, dtype, engine, converters, true_values, false_values, skipinitialspace, skiprows, skipfooter, nrows, na_values, keep_default_na, na_filter, verbose, skip_blank_lines, parse_dates, infer_datetime_format, keep_date_col, date_parser, dayfirst, cache_dates, iterator, chunksize, compression, thousands, decimal, lineterminator, quotechar, quoting, doublequote, escapechar, comment, encoding, dialect, error_bad_lines, warn_bad_lines, delim_whitespace, low_memory, memory_map, float_precision)\u001b[0m\n\u001b[0;32m    674\u001b[0m         )\n\u001b[0;32m    675\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 676\u001b[1;33m         \u001b[1;32mreturn\u001b[0m \u001b[0m_read\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mfilepath_or_buffer\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mkwds\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    677\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    678\u001b[0m     \u001b[0mparser_f\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m__name__\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mname\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\anaconda\\lib\\site-packages\\pandas\\io\\parsers.py\u001b[0m in \u001b[0;36m_read\u001b[1;34m(filepath_or_buffer, kwds)\u001b[0m\n\u001b[0;32m    446\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    447\u001b[0m     \u001b[1;31m# Create the parser.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 448\u001b[1;33m     \u001b[0mparser\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mTextFileReader\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mfp_or_buf\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    449\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    450\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0mchunksize\u001b[0m \u001b[1;32mor\u001b[0m \u001b[0miterator\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\anaconda\\lib\\site-packages\\pandas\\io\\parsers.py\u001b[0m in \u001b[0;36m__init__\u001b[1;34m(self, f, engine, **kwds)\u001b[0m\n\u001b[0;32m    878\u001b[0m             \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0moptions\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m\"has_index_names\"\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mkwds\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m\"has_index_names\"\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    879\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 880\u001b[1;33m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_make_engine\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mengine\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    881\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    882\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mclose\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\anaconda\\lib\\site-packages\\pandas\\io\\parsers.py\u001b[0m in \u001b[0;36m_make_engine\u001b[1;34m(self, engine)\u001b[0m\n\u001b[0;32m   1112\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0m_make_engine\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mengine\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;34m\"c\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1113\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mengine\u001b[0m \u001b[1;33m==\u001b[0m \u001b[1;34m\"c\"\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1114\u001b[1;33m             \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_engine\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mCParserWrapper\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mf\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0moptions\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1115\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1116\u001b[0m             \u001b[1;32mif\u001b[0m \u001b[0mengine\u001b[0m \u001b[1;33m==\u001b[0m \u001b[1;34m\"python\"\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\anaconda\\lib\\site-packages\\pandas\\io\\parsers.py\u001b[0m in \u001b[0;36m__init__\u001b[1;34m(self, src, **kwds)\u001b[0m\n\u001b[0;32m   1889\u001b[0m         \u001b[0mkwds\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m\"usecols\"\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0musecols\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1890\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1891\u001b[1;33m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_reader\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mparsers\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mTextReader\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0msrc\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1892\u001b[0m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0munnamed_cols\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_reader\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0munnamed_cols\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1893\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mpandas\\_libs\\parsers.pyx\u001b[0m in \u001b[0;36mpandas._libs.parsers.TextReader.__cinit__\u001b[1;34m()\u001b[0m\n",
      "\u001b[1;32mpandas\\_libs\\parsers.pyx\u001b[0m in \u001b[0;36mpandas._libs.parsers.TextReader._setup_parser_source\u001b[1;34m()\u001b[0m\n",
      "\u001b[1;31mFileNotFoundError\u001b[0m: [Errno 2] File ./data/weather1.csv does not exist: './data/weather1.csv'"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import os\n",
    "\n",
    "def merge_data(pv_data,weather_data):\n",
    "    weather_data = pd.read_csv(weather_data)\n",
    "    files= os.listdir(pv_data)\n",
    "    for file in files:\n",
    "        path = pv_data + file\n",
    "        pvdata = pd.read_csv(path)\n",
    "        train_data = pd.merge(pvdata, weather_data, how='left', on='time') #光伏数据与天气数据根据时间左连接合并\n",
    "        train_data.insert(5,'hour',(train_data['data_date'])%100)\n",
    "        #train_data = train_data.fillna(method ='ffill')\n",
    "        pvdata = pd.DataFrame(pvdata.sort_values(by=['ycsb','data_date']))\n",
    "        train_data.to_csv('./data/train_data_test_all/'+file, encoding='utf_8_sig',index=None)\n",
    "        #print('完成天气合并:',path)\n",
    "    return train_data\n",
    "\n",
    "def read_merger_data():\n",
    "    file_dir = './data/train_data_test_all/'\n",
    "    all_csv_list = os.listdir(file_dir)\n",
    "    for single_csv in all_csv_list:\n",
    "        print(single_csv)\n",
    "        single_data_frame = pd.read_csv(os.path.join(file_dir, single_csv))\n",
    "        \n",
    "        if single_csv == all_csv_list[0]:\n",
    "            all_data_frame = single_data_frame\n",
    "        else:\n",
    "            all_data_frame = pd.concat([all_data_frame, single_data_frame],axis=0)\n",
    "    all_data_frame= pd.DataFrame(all_data_frame.sort_values(by=['ycsb','data_date']))\n",
    "    all_data_frame.to_csv('./data/all_train_data_all.csv')\n",
    "    print('合并完成')\n",
    "\n",
    "if __name__=='__main__':\n",
    "    pv_data = './data/date_data_all/'\n",
    "    weather_data = './data/weather1.csv'\n",
    "    merge_data(pv_data,weather_data)\n",
    "    read_merger_data()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 用户数据数量查看"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "d = dict()\n",
    "for i,num in enumerate(yh.tolist()):\n",
    "    if num not in d:\n",
    "        d[num] = 1\n",
    "    else:\n",
    "        d[num] = d[num] + 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{256: 73,\n",
       " 240: 31,\n",
       " 272: 8,\n",
       " 224: 7,\n",
       " 288: 19,\n",
       " 64: 3,\n",
       " 80: 1,\n",
       " 269: 1,\n",
       " 47: 1,\n",
       " 16: 1,\n",
       " 48: 1}"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "d"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "256，288        89条\n",
    "240，256，288    \n",
    "\n",
    "\n",
    "\n",
    "根据用户数据量，使用"
   ]
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
